REGISTER tw.jar;
log0 = LOAD '/hnbc/' USING PigStorage('\u0001') AS (line:chararray);
log1 = FOREACH log0 GENERATE FLATTEN(com.tw.udf.LogFormat(line)) as 
																	(datetime:chararray,
																	requesttime:chararray,
																	domain:chararray,
																	requesturl:chararray,
																	srcip:chararray,
																	srcprovince:chararray,
																	srcisp:chararray,
																	srciprange:chararray,
																	destip:chararray,
																	destprovince:chararray,
																	destisp:chararray,
																	destiprange:chararray,
																	type:chararray,
																	os:chararray,
																	browser:chararray,
																	referer:chararray,
																	phone:chararray);
log2 = GROUP log1 BY os;
log3 = FOREACH log2 	GENERATE group,COUNT(log1) as num;
logs0 = ORDER log3 BY num DESC;
STORE logs0 INTO '/tw/warehouse/os/' USING PigStorage();	
	
log4 = GROUP log1 BY browser;
log5 = FOREACH log4 	GENERATE group,COUNT(log1) as num;
logs1 = ORDER log5 BY num DESC;
STORE logs1 INTO '/tw/warehouse/browser/' USING PigStorage();

log6 = GROUP log1 BY phone;
log7 = FOREACH log6 	GENERATE group,COUNT(log1);
logs2 = ORDER log7 BY num DESC;
STORE logs2 INTO '/tw/warehouse/phone/' USING PigStorage();

log8 = GROUP log1 BY domain;
log9 = FOREACH log8 	GENERATE group,COUNT(log1) as num;
logs3 = ORDER log9 BY num DESC;
logl0 = LIMIT logs3 10000;
STORE logl0 INTO '/tw/warehouse/domain/' USING PigStorage();

log10 = GROUP log1 BY requesturl;
log11 = FOREACH log10 {D = DISTINCT log1.srcip; GENERATE group as requesturl, COUNT(D) AS uvs;};
logs4 = ORDER log11 BY uvs DESC;
logl1 = LIMIT logs4 10000;
STORE logl1 INTO '/tw/warehouse/urluvs/' USING PigStorage();

log12 = GROUP log1 BY referer;
log13 = FOREACH log12 GENERATE group,COUNT(log1) as num;
logs5 = ORDER log13 BY num DESC;
logl2 = LIMIT logs5 10000;
STORE logl2 INTO '/tw/warehouse/referer/' USING PigStorage();


						